'''
Created on Aug 28, 2012

@author: kingsfield
'''
import string
import Util

exclude = set(string.punctuation)
print exclude

def wordclean(word):
    res = word.strip()
    res = res.lower()
    res = ''.join(ch for ch in res if ch not in exclude)
    res = res.strip()
    return res

def is_legal(word):
    if word.find(' ') > 0 or len(word) <= 0:
        return False
    else:
        return True

if __name__ == '__main__':
    filepath = '/home/kingsfield/Desktop/kaggle/kaggle/train.csv'
    sum_line = 1865270
    
    uidmap = dict()
    itemmap = dict()
    catmap = dict()
    wordmap = dict()
    cat_item_map = dict()
    cat_item_count = dict()
    cat_word_map = dict()
    cat_word_count = dict()
    max_uid = 1
    max_iid = 1
    max_cid = 1
    max_wid = 1
    
    uidcount = dict()
    itemcount = dict()
    catcount = dict()
    wordcount = dict()
    with open(filepath) as fr:
        line = fr.readline()
        while True:
            line = fr.readline()
            if not line:
                break
            ss = line.split(',')
            if len(ss) != 6:
                print line
            'uid'
            uid = ss[0]
            if not uid in uidmap:
                uidmap[uid] = max_uid
                max_uid += 1
            uidcount[uid] = uidcount.get(uid, 0) + 1
            'iid'
            iid = ss[1]
            if not iid in itemmap:
                itemmap[iid] = max_iid
                max_iid += 1
            itemcount[iid] = itemcount.get(iid, 0) + 1
            'cid'
            cid = ss[2]
            if not cid in catmap:
                catmap[cid] = max_cid
                cat_item_map[max_cid] = set()
                cat_item_count[max_cid] = dict()
                cat_word_map[max_cid] = set()
                cat_word_count[max_cid] = dict()
                max_cid += 1
            catcount[cid] = catcount.get(cid, 0) + 1
            cat_item_count[catmap[cid]][itemmap[iid]] = cat_item_count[catmap[cid]].get(itemmap[iid], 0) + 1 
            'wid'
            words = ss[3]
            ws = Util.wordutil.getwords(words)
            for wid in ws:
                if not wid in wordmap:
                    wordmap[wid] = wordmap.get(wid, max_wid)
                    max_wid += 1
                wordcount[wid] = wordcount.get(wid, 0) + 1
                cat_word_map[catmap[cid]].add(wordmap[wid])
                cat_word_count[catmap[cid]][wordmap[wid]] = cat_word_count[catmap[cid]].get(wordmap[wid], 0) + 1  
            'cid and iid map'
            cat_item_map[catmap[cid]].add(itemmap[iid])
            
    print 'get %d users, users\' max query = %d, users\' avg query = %f' % (max_uid - 1, max(uidcount.values()), sum_line * 1.0 / max_uid)
    print 'get %d items, items\' max query = %d, items\' avg query = %f' % (max_iid - 1 , max(itemcount.values()), sum_line * 1.0 / max_iid)
    print 'get %d category, cats\' max query = %d, cats\' avg query = %f' % (max_cid - 1, max(catcount.values()), sum_line * 1.0 / max_cid)
    print 'get %d words, words\' max query = %d, words\' avg query = %f' % (max_wid - 1, max(wordcount.values()), sum_line * 1.0 / max_wid)
    
    with open('/home/kingsfield/Desktop/kaggle/kaggle/idmap/wordmap', 'w') as fw:
        for w in wordmap:
            fw.write('%s %s\n' % (w, wordmap[w]))
    with open('/home/kingsfield/Desktop/kaggle/kaggle/idmap/categorymap', 'w') as fw:
        for c in catmap:
            fw.write('%s %s\n' % (c, catmap[c]))
    with open('/home/kingsfield/Desktop/kaggle/kaggle/idmap/itemmap', 'w') as fw:
        for i in itemmap:
            fw.write('%s %s\n' % (i, itemmap[i]))
    with open('/home/kingsfield/Desktop/kaggle/kaggle/idmap/catitemmap', 'w') as fw:
        for cid in cat_item_count:
            for iid in cat_item_count[cid]:
                fw.write('%s %s %d\n' % (cid, iid, cat_item_count[cid][iid]))
    with open('/home/kingsfield/Desktop/kaggle/kaggle/idmap/catwordmap', 'w') as fw:
        for cid in cat_word_count:
            for wid in cat_word_count[cid]:
                fw.write('%s %s %d\n' % (cid, wid, cat_word_count[cid][wid]))
#    word_rank = sorted(wordcount.items(), key=lambda x:x[1], reverse=True)
#    top50 = word_rank[0:50]
#    for line in top50:
#        print str(line)
    print 'done'
            
